Match plasma & serum samples
import pandas as pd
samples = pd.read_csv('data/sample_sheet.csv')
# must be Healthy Control Study and must pass MiSeq QC
samples = samples.loc[(samples['Study'] == 'Healthy Controls') & (samples['MISEQ.QC.PASS'] == 'PASS')]
samples = samples.set_index('MT.Unique.ID').sort_values(by=['Participant.ID', 'Source'])
# capitalize & strip whitespace for consistency
for column in ['Gender', 'Race', 'Source']:
samples[column] = samples[column].str.capitalize()
samples[column] = samples[column].str.strip()
# use correct ontology terms
race_ontology = {'Asian': 'Asian',
'Black or african american': 'African_American',
'Mixed/asian & white': 'Multiracial',
'Mixed/asian &black': 'Multiracial',
'Mixed/black, white, asian': 'Multiracial',
'Native hawiian or other pacific islander': 'Pacific_Islander',
'Pacific islander': 'Pacific_Islander',
'White': 'White'}
for id in samples.index:
race = samples.at[id, 'Race']
samples.at[id, 'Race'] = race_ontology[race] if race in race_ontology else 'Multiracial'
# get matched plasma & serum samples
mir_counts = pd.read_csv("data/get_canonical/canon_mir_counts.csv", index_col=0)
samples.index = pd.Index(['X' + str(row) for row in samples.index])
serum_part_ids = set(samples.loc[samples['Source'] =='Serum']['Participant.ID'])
matched_samples = samples.loc[samples['Participant.ID'].isin(serum_part_ids)]
matched_mir_counts = mir_counts[matched_samples.index]
matched_samples.to_csv('data/matched_plasma-serum_samples.csv')
matched_mir_counts.to_csv('data/matched_plasma-serum_mir_counts.csv')
Load the sample data and miRNA counts
samples <- read.csv('data/matched_plasma-serum_samples.csv')
samples <- subset(samples, Library.Generation.Set != "SetRecheck" ) # exclude SetRecheck samples
# sample X11 is outlier -- remove it and its match
outlier_id <- samples[samples$X=="X11",]$Participant.ID
samples <- samples[samples$Participant.ID != outlier_id,]
counts <- read.csv('data/matched_plasma-serum_mir_counts.csv')
samples$Participant.ID <- factor(samples$Participant.ID) # ID is categorical, not numerical
samples$Sample.ID <- factor(samples$Sample.ID)
rownames(counts) = counts$X
rownames(samples) = samples$X
counts$X <- NULL # remove extra column
counts <- counts[,rownames(samples)]
head(samples)
head(counts)
Filter
library(edgeR)
design <- model.matrix(~0+Source+Race+Gender+Age, samples)
dge = DGEList(counts = counts, samples = samples)
# require miRNAs to have CPM > 1 in at least 2 samples
countsPerMillion <- edgeR::cpm(dge)
countCheck <- countsPerMillion > 1
head(countCheck)
X1 X107 X2 X108 X14 X110 X15 X18 X21 X113 X22 X114 X26 X115 X40 X116 X46 X119 X48 X120 X52
hsa-let-7a-3p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE FALSE TRUE TRUE TRUE FALSE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
hsa-let-7a-5p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
hsa-let-7b-3p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE FALSE TRUE TRUE TRUE
hsa-let-7b-5p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
hsa-let-7c-3p FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
hsa-let-7c-5p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
X121 X56 X122 X59 X123 X60 X124 X62 X125 X64 X126 X66 X127 X67 X128 X68 X129 X69 X130 X131 X77
hsa-let-7a-3p TRUE TRUE FALSE TRUE FALSE TRUE FALSE TRUE FALSE TRUE TRUE TRUE TRUE TRUE FALSE TRUE FALSE TRUE FALSE TRUE TRUE
hsa-let-7a-5p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
hsa-let-7b-3p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
hsa-let-7b-5p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
hsa-let-7c-3p FALSE FALSE FALSE FALSE FALSE FALSE FALSE TRUE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
hsa-let-7c-5p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
X132 X133 X88 X134 X89 X135 X90 X136 X93 X137 X138
hsa-let-7a-3p FALSE FALSE TRUE TRUE TRUE FALSE TRUE TRUE TRUE TRUE TRUE
hsa-let-7a-5p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
hsa-let-7b-3p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
hsa-let-7b-5p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
hsa-let-7c-3p FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE TRUE FALSE FALSE
hsa-let-7c-5p TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE
keep <- which(rowSums(countCheck) >= 2)
dge <- dge[keep,]
Explore variance
library(SingleCellExperiment)
library(scater)
reads_sce <- SingleCellExperiment(assays=list(counts=dge$counts), colData=dge$samples)
# remove unexpressed miRNAs
keep_feature <- rowSums(counts(reads_sce) > 0) > 0
reads_sce <- reads_sce[keep_feature, ]
reads_sce <- calculateQCMetrics(reads_sce)
Note that the names of some metrics have changed, see 'Renamed metrics' in ?calculateQCMetrics.
Old names are currently maintained for back-compatibility, but may be removed in future releases.
# create separate plasma & serum SCEs
plasma_samples <- subset(colData(reads_sce), Source == "Plasma")
plasma_counts <- as.data.frame(counts(reads_sce))[rownames(plasma_samples)]
plasma_sce <- SingleCellExperiment(assays=list(counts=as.matrix(plasma_counts)), colData=plasma_samples)
plasma_sce <- calculateQCMetrics(plasma_sce)
Note that the names of some metrics have changed, see 'Renamed metrics' in ?calculateQCMetrics.
Old names are currently maintained for back-compatibility, but may be removed in future releases.
serum_samples <- subset(colData(reads_sce), Source == "Serum")
serum_counts <- as.data.frame(counts(reads_sce))[rownames(serum_samples)]
serum_sce <- SingleCellExperiment(assays=list(counts=as.matrix(serum_counts)), colData=serum_samples)
serum_sce <- calculateQCMetrics(serum_sce)
Note that the names of some metrics have changed, see 'Renamed metrics' in ?calculateQCMetrics.
Old names are currently maintained for back-compatibility, but may be removed in future releases.
# log transform
cpm(reads_sce) <- calculateCPM(reads_sce)
reads_sce <- normalize(reads_sce)
using library sizes as size factors
logcounts(reads_sce) <- log2(calculateCPM(reads_sce) + 1)
# visualize
hist(reads_sce$total_counts, breaks=100) # counts per sample

hist(reads_sce$total_features, breaks=100) # counts per miRNA

plotQC(reads_sce, type = "highest-expression") + ggtitle("Plasma & Serum")

plotQC(plasma_sce, type = "highest-expression") + ggtitle("Plasma")

plotQC(serum_sce, type = "highest-expression") + ggtitle("Serum")

plotQC(reads_sce, type="explanatory-variables", variables=c("Index", "Participant.ID", "Collection.Date", "Library.Generation.Set", "MiSeq.QC.Run", "Source", 'total_features', "Age", "Race", "Sex"))
variable Sex not found in colData(object).
Please make sure colData(object)[, variable] exists. This variable will not be plotted.

Examine sources of variation
plotPCA(reads_sce, exprs_values = "logcounts", colour_by = "Library.Generation.Set", size_by = "total_features", shape_by = "Source")
non-plotting arguments like 'exprs_values' should go in 'run_args'

for (var in c("total_features", "Age", "Library.Generation.Set", "Index", "Participant.ID", "Source", "Race", "Gender", "Collection.Date")) {
print(
plotQC(reads_sce, type = "find-pcs", exprs_values = "logcounts", variable = var)
) + ggtitle(var)
}









Normalize & Remove unwanted sources of variation
library(RUVSeq)
library(ggplot2)
library(mvoutlier)
dge <- calcNormFactors(dge)
dge <- estimateGLMCommonDisp(dge, design)
dge <- estimateGLMTagwiseDisp(dge, design)
fit <- glmFit(dge, design)
lrt <- glmLRT(fit, coef=2)
res <- residuals(fit, type="deviance")
set <- newSeqExpressionSet(dge$counts, phenoData=dge$samples)
for(k in 1:10) {
ruvr_set <- RUVr(set, row.names(dge), k=k, res)
assay(reads_sce, paste("RUVr k=", toString(k))) <- log2(t(t(assayData(ruvr_set)$normalizedCounts) / colSums(assayData(ruvr_set)$normalizedCounts) * 1e6) + 1)
}
for(n in assayNames(reads_sce)) {
print(
plotPCA(
reads_sce,
colour_by = "Library.Generation.Set",
size_by = "total_features",
shape_by = "Source",
exprs_values = n
) + ggtitle(n)
)
}
non-plotting arguments like 'exprs_values' should go in 'run_args'













Detect outliers
reads_sce <- runPCA(reads_sce, use_coldata = TRUE, detect_outliers = TRUE)
failed to find 'pct_counts_feature_control' in column metadatafailed to find 'total_features_by_counts_feature_control' in column metadatafailed to find 'log10_total_counts_endogenous' in column metadatafailed to find 'log10_total_counts_feature_control' in column metadata
outliers <- colnames(reads_sce)[reads_sce$outlier]
head(outliers)
character(0)
Examine sources of variance after removing unwanted variation
for (var in c("total_features", "Age", "Library.Generation.Set", "Index", "Participant.ID", "Source", "Race", "Gender", "Collection.Date")) {
print(
plotQC(reads_sce, type = "find-pcs", exprs_values = "RUVr k= 2", variable = var)
) + ggtitle(var)
}









Visualize top plasma-vs-serum miRNAs


Run the DESeq2 pipeline & Visualize results
library(DESeq2) # TODO: use filtered & RUVSeq-corrected data
deseq2Data <- DESeqDataSetFromMatrix(countData = counts, design = ~ Participant.ID + Source, colData = samples)
deseq2Data <- DESeq(deseq2Data)
results <- results(deseq2Data, cooksCutoff=FALSE, independentFiltering=FALSE)
head(as.data.frame(results))
Create a heatmap of sample-to-sample distances
sampleDists <- dist(t(assay(vstData))) # compute sample-to-sample distances
sampleDistMatrix <- as.matrix(sampleDists)
rownames(sampleDistMatrix) <- paste(vstData$Participant.ID, vstData$Source, sep=" - ")
colnames(sampleDistMatrix) <- NULL
colors <- colorRampPalette( rev(brewer.pal(9, "Blues")) )(255)
pheatmap(sampleDistMatrix,
clustering_distance_rows=sampleDists,
clustering_distance_cols=sampleDists,
col=colors)
Create a PCA plot
pcaData <- plotPCA(vstData, intgroup=c("Source"), returnData=TRUE)
percentVar <- round(100 * attr(pcaData, "percentVar"))
ggplot(pcaData, aes(PC1, PC2, color=Source)) +
geom_point(size=3) +
xlab(paste0("PC1: ",percentVar[1],"% variance")) +
ylab(paste0("PC2: ",percentVar[2],"% variance")) +
coord_fixed()
save.image("diff_expr_plasma_vs_serum.RData")
LS0tCnRpdGxlOiAiRGlmZmVyZW50aWFsIGV4cHJlc3Npb24gYW5hbHlzaXMgdXNpbmcgREVTZXEyIGZvciBtYXRjaGVkIHBsYXNtYSAmIHNlcnVtIHNhbXBsZXMiCm91dHB1dDogaHRtbF9ub3RlYm9vawplZGl0b3Jfb3B0aW9uczogCiAgY2h1bmtfb3V0cHV0X3R5cGU6IGlubGluZQotLS0KIyBNYXRjaCBwbGFzbWEgJiBzZXJ1bSBzYW1wbGVzCmBgYHtweXRob24zfQppbXBvcnQgcGFuZGFzIGFzIHBkCnNhbXBsZXMgPSBwZC5yZWFkX2NzdignZGF0YS9zYW1wbGVfc2hlZXQuY3N2JykKIyBtdXN0IGJlIEhlYWx0aHkgQ29udHJvbCBTdHVkeSBhbmQgbXVzdCBwYXNzIE1pU2VxIFFDCnNhbXBsZXMgPSBzYW1wbGVzLmxvY1soc2FtcGxlc1snU3R1ZHknXSA9PSAnSGVhbHRoeSBDb250cm9scycpICYgKHNhbXBsZXNbJ01JU0VRLlFDLlBBU1MnXSA9PSAnUEFTUycpXSAgCnNhbXBsZXMgPSBzYW1wbGVzLnNldF9pbmRleCgnTVQuVW5pcXVlLklEJykuc29ydF92YWx1ZXMoYnk9WydQYXJ0aWNpcGFudC5JRCcsICdTb3VyY2UnXSkKIyBjYXBpdGFsaXplICYgc3RyaXAgd2hpdGVzcGFjZSBmb3IgY29uc2lzdGVuY3kKZm9yIGNvbHVtbiBpbiBbJ0dlbmRlcicsICdSYWNlJywgJ1NvdXJjZSddOgogICAgc2FtcGxlc1tjb2x1bW5dID0gc2FtcGxlc1tjb2x1bW5dLnN0ci5jYXBpdGFsaXplKCkKICAgIHNhbXBsZXNbY29sdW1uXSA9IHNhbXBsZXNbY29sdW1uXS5zdHIuc3RyaXAoKQojIHVzZSBjb3JyZWN0IG9udG9sb2d5IHRlcm1zCnJhY2Vfb250b2xvZ3kgPSB7J0FzaWFuJzogJ0FzaWFuJywKICdCbGFjayBvciBhZnJpY2FuIGFtZXJpY2FuJzogJ0FmcmljYW5fQW1lcmljYW4nLAogJ01peGVkL2FzaWFuICYgd2hpdGUnOiAnTXVsdGlyYWNpYWwnLAogJ01peGVkL2FzaWFuICZibGFjayc6ICdNdWx0aXJhY2lhbCcsCiAnTWl4ZWQvYmxhY2ssIHdoaXRlLCBhc2lhbic6ICdNdWx0aXJhY2lhbCcsCiAnTmF0aXZlIGhhd2lpYW4gb3Igb3RoZXIgcGFjaWZpYyBpc2xhbmRlcic6ICdQYWNpZmljX0lzbGFuZGVyJywKICdQYWNpZmljIGlzbGFuZGVyJzogJ1BhY2lmaWNfSXNsYW5kZXInLAogJ1doaXRlJzogJ1doaXRlJ30KZm9yIGlkIGluIHNhbXBsZXMuaW5kZXg6CiAgICByYWNlID0gc2FtcGxlcy5hdFtpZCwgJ1JhY2UnXQogICAgc2FtcGxlcy5hdFtpZCwgJ1JhY2UnXSA9IHJhY2Vfb250b2xvZ3lbcmFjZV0gaWYgcmFjZSBpbiByYWNlX29udG9sb2d5IGVsc2UgJ011bHRpcmFjaWFsJwojIGdldCBtYXRjaGVkIHBsYXNtYSAmIHNlcnVtIHNhbXBsZXMKbWlyX2NvdW50cyA9IHBkLnJlYWRfY3N2KCJkYXRhL2dldF9jYW5vbmljYWwvY2Fub25fbWlyX2NvdW50cy5jc3YiLCBpbmRleF9jb2w9MCkKc2FtcGxlcy5pbmRleCA9IHBkLkluZGV4KFsnWCcgKyBzdHIocm93KSBmb3Igcm93IGluIHNhbXBsZXMuaW5kZXhdKQpzZXJ1bV9wYXJ0X2lkcyA9IHNldChzYW1wbGVzLmxvY1tzYW1wbGVzWydTb3VyY2UnXSA9PSdTZXJ1bSddWydQYXJ0aWNpcGFudC5JRCddKQptYXRjaGVkX3NhbXBsZXMgPSBzYW1wbGVzLmxvY1tzYW1wbGVzWydQYXJ0aWNpcGFudC5JRCddLmlzaW4oc2VydW1fcGFydF9pZHMpXQptYXRjaGVkX21pcl9jb3VudHMgPSBtaXJfY291bnRzW21hdGNoZWRfc2FtcGxlcy5pbmRleF0KbWF0Y2hlZF9zYW1wbGVzLnRvX2NzdignZGF0YS9tYXRjaGVkX3BsYXNtYS1zZXJ1bV9zYW1wbGVzLmNzdicpCm1hdGNoZWRfbWlyX2NvdW50cy50b19jc3YoJ2RhdGEvbWF0Y2hlZF9wbGFzbWEtc2VydW1fbWlyX2NvdW50cy5jc3YnKQpgYGAKIyMgTG9hZCB0aGUgc2FtcGxlIGRhdGEgYW5kIG1pUk5BIGNvdW50cwpgYGB7cn0Kc2FtcGxlcyA8LSByZWFkLmNzdignZGF0YS9tYXRjaGVkX3BsYXNtYS1zZXJ1bV9zYW1wbGVzLmNzdicpCnNhbXBsZXMgPC0gc3Vic2V0KHNhbXBsZXMsIExpYnJhcnkuR2VuZXJhdGlvbi5TZXQgIT0gIlNldFJlY2hlY2siICkgICMgZXhjbHVkZSBTZXRSZWNoZWNrIHNhbXBsZXMKIyBzYW1wbGUgWDExIGlzIG91dGxpZXIgLS0gcmVtb3ZlIGl0IGFuZCBpdHMgbWF0Y2gKb3V0bGllcl9pZCA8LSBzYW1wbGVzW3NhbXBsZXMkWD09IlgxMSIsXSRQYXJ0aWNpcGFudC5JRApzYW1wbGVzIDwtIHNhbXBsZXNbc2FtcGxlcyRQYXJ0aWNpcGFudC5JRCAhPSBvdXRsaWVyX2lkLF0KY291bnRzIDwtIHJlYWQuY3N2KCdkYXRhL21hdGNoZWRfcGxhc21hLXNlcnVtX21pcl9jb3VudHMuY3N2JykKc2FtcGxlcyRQYXJ0aWNpcGFudC5JRCA8LSBmYWN0b3Ioc2FtcGxlcyRQYXJ0aWNpcGFudC5JRCkgICMgSUQgaXMgY2F0ZWdvcmljYWwsIG5vdCBudW1lcmljYWwKc2FtcGxlcyRTYW1wbGUuSUQgPC0gZmFjdG9yKHNhbXBsZXMkU2FtcGxlLklEKQpyb3duYW1lcyhjb3VudHMpID0gY291bnRzJFgKcm93bmFtZXMoc2FtcGxlcykgPSBzYW1wbGVzJFgKY291bnRzJFggPC0gTlVMTCAgIyByZW1vdmUgZXh0cmEgY29sdW1uCmNvdW50cyA8LSBjb3VudHNbLHJvd25hbWVzKHNhbXBsZXMpXQpoZWFkKHNhbXBsZXMpCmhlYWQoY291bnRzKQpgYGAKIyMgRmlsdGVyCmBgYHtyfQpsaWJyYXJ5KGVkZ2VSKQpkZXNpZ24gPC0gbW9kZWwubWF0cml4KH4wK1NvdXJjZStSYWNlK0dlbmRlcitBZ2UsIHNhbXBsZXMpCmRnZSA9IERHRUxpc3QoY291bnRzID0gY291bnRzLCBzYW1wbGVzID0gc2FtcGxlcykKIyByZXF1aXJlIG1pUk5BcyB0byBoYXZlIENQTSA+IDEgaW4gYXQgbGVhc3QgMiBzYW1wbGVzCmNvdW50c1Blck1pbGxpb24gPC0gZWRnZVI6OmNwbShkZ2UpCmNvdW50Q2hlY2sgPC0gY291bnRzUGVyTWlsbGlvbiA+IDEKaGVhZChjb3VudENoZWNrKQprZWVwIDwtIHdoaWNoKHJvd1N1bXMoY291bnRDaGVjaykgPj0gMikgCmRnZSA8LSBkZ2Vba2VlcCxdCmBgYAojIyBFeHBsb3JlIHZhcmlhbmNlCmBgYHtyfQpsaWJyYXJ5KFNpbmdsZUNlbGxFeHBlcmltZW50KQpsaWJyYXJ5KHNjYXRlcikKcmVhZHNfc2NlIDwtIFNpbmdsZUNlbGxFeHBlcmltZW50KGFzc2F5cz1saXN0KGNvdW50cz1kZ2UkY291bnRzKSwgIGNvbERhdGE9ZGdlJHNhbXBsZXMpCiMgcmVtb3ZlIHVuZXhwcmVzc2VkIG1pUk5BcwprZWVwX2ZlYXR1cmUgPC0gcm93U3Vtcyhjb3VudHMocmVhZHNfc2NlKSA+IDApID4gMApyZWFkc19zY2UgPC0gcmVhZHNfc2NlW2tlZXBfZmVhdHVyZSwgXQpyZWFkc19zY2UgPC0gY2FsY3VsYXRlUUNNZXRyaWNzKHJlYWRzX3NjZSkKIyBjcmVhdGUgc2VwYXJhdGUgcGxhc21hICYgc2VydW0gU0NFcwpwbGFzbWFfc2FtcGxlcyA8LSBzdWJzZXQoY29sRGF0YShyZWFkc19zY2UpLCBTb3VyY2UgPT0gIlBsYXNtYSIpCnBsYXNtYV9jb3VudHMgPC0gYXMuZGF0YS5mcmFtZShjb3VudHMocmVhZHNfc2NlKSlbcm93bmFtZXMocGxhc21hX3NhbXBsZXMpXQpwbGFzbWFfc2NlIDwtIFNpbmdsZUNlbGxFeHBlcmltZW50KGFzc2F5cz1saXN0KGNvdW50cz1hcy5tYXRyaXgocGxhc21hX2NvdW50cykpLCBjb2xEYXRhPXBsYXNtYV9zYW1wbGVzKQpwbGFzbWFfc2NlIDwtIGNhbGN1bGF0ZVFDTWV0cmljcyhwbGFzbWFfc2NlKQpzZXJ1bV9zYW1wbGVzIDwtIHN1YnNldChjb2xEYXRhKHJlYWRzX3NjZSksIFNvdXJjZSA9PSAiU2VydW0iKQpzZXJ1bV9jb3VudHMgPC0gYXMuZGF0YS5mcmFtZShjb3VudHMocmVhZHNfc2NlKSlbcm93bmFtZXMoc2VydW1fc2FtcGxlcyldCnNlcnVtX3NjZSA8LSBTaW5nbGVDZWxsRXhwZXJpbWVudChhc3NheXM9bGlzdChjb3VudHM9YXMubWF0cml4KHNlcnVtX2NvdW50cykpLCBjb2xEYXRhPXNlcnVtX3NhbXBsZXMpCnNlcnVtX3NjZSA8LSBjYWxjdWxhdGVRQ01ldHJpY3Moc2VydW1fc2NlKQojIGxvZyB0cmFuc2Zvcm0KY3BtKHJlYWRzX3NjZSkgPC0gY2FsY3VsYXRlQ1BNKHJlYWRzX3NjZSkKcmVhZHNfc2NlIDwtIG5vcm1hbGl6ZShyZWFkc19zY2UpCmxvZ2NvdW50cyhyZWFkc19zY2UpIDwtIGxvZzIoY2FsY3VsYXRlQ1BNKHJlYWRzX3NjZSkgKyAxKQojIHZpc3VhbGl6ZQpoaXN0KHJlYWRzX3NjZSR0b3RhbF9jb3VudHMsIGJyZWFrcz0xMDApICAjIGNvdW50cyBwZXIgc2FtcGxlCmhpc3QocmVhZHNfc2NlJHRvdGFsX2ZlYXR1cmVzLCBicmVha3M9MTAwKSAgIyBjb3VudHMgcGVyIG1pUk5BCnBsb3RRQyhyZWFkc19zY2UsIHR5cGUgPSAiaGlnaGVzdC1leHByZXNzaW9uIikgKyBnZ3RpdGxlKCJQbGFzbWEgJiBTZXJ1bSIpCnBsb3RRQyhwbGFzbWFfc2NlLCB0eXBlID0gImhpZ2hlc3QtZXhwcmVzc2lvbiIpICsgZ2d0aXRsZSgiUGxhc21hIikKcGxvdFFDKHNlcnVtX3NjZSwgdHlwZSA9ICJoaWdoZXN0LWV4cHJlc3Npb24iKSArIGdndGl0bGUoIlNlcnVtIikKcGxvdFFDKHJlYWRzX3NjZSwgdHlwZT0iZXhwbGFuYXRvcnktdmFyaWFibGVzIiwgdmFyaWFibGVzPWMoIkluZGV4IiwgIlBhcnRpY2lwYW50LklEIiwgIkNvbGxlY3Rpb24uRGF0ZSIsICJMaWJyYXJ5LkdlbmVyYXRpb24uU2V0IiwgIk1pU2VxLlFDLlJ1biIsICJTb3VyY2UiLCAndG90YWxfZmVhdHVyZXMnLCAiQWdlIiwgIlJhY2UiLCAiU2V4IikpCmBgYAojIyBFeGFtaW5lIHNvdXJjZXMgb2YgdmFyaWF0aW9uCmBgYHtyfQpwbG90UENBKHJlYWRzX3NjZSwgZXhwcnNfdmFsdWVzID0gImxvZ2NvdW50cyIsIGNvbG91cl9ieSA9ICJMaWJyYXJ5LkdlbmVyYXRpb24uU2V0Iiwgc2l6ZV9ieSA9ICJ0b3RhbF9mZWF0dXJlcyIsIHNoYXBlX2J5ID0gIlNvdXJjZSIpCmZvciAodmFyIGluIGMoInRvdGFsX2ZlYXR1cmVzIiwgIkFnZSIsICJMaWJyYXJ5LkdlbmVyYXRpb24uU2V0IiwgIkluZGV4IiwgIlBhcnRpY2lwYW50LklEIiwgIlNvdXJjZSIsICJSYWNlIiwgIkdlbmRlciIsICJDb2xsZWN0aW9uLkRhdGUiKSkgewogIHByaW50KAogICAgcGxvdFFDKHJlYWRzX3NjZSwgdHlwZSA9ICJmaW5kLXBjcyIsIGV4cHJzX3ZhbHVlcyA9ICJsb2djb3VudHMiLCB2YXJpYWJsZSA9IHZhcikKICAgICkgICsgZ2d0aXRsZSh2YXIpIAogIH0KYGBgCgojIyBOb3JtYWxpemUgJiBSZW1vdmUgdW53YW50ZWQgc291cmNlcyBvZiB2YXJpYXRpb24KYGBge3J9CmxpYnJhcnkoUlVWU2VxKQpsaWJyYXJ5KGdncGxvdDIpCmxpYnJhcnkobXZvdXRsaWVyKQpkZ2UgPC0gY2FsY05vcm1GYWN0b3JzKGRnZSkKZGdlIDwtIGVzdGltYXRlR0xNQ29tbW9uRGlzcChkZ2UsIGRlc2lnbikKZGdlIDwtIGVzdGltYXRlR0xNVGFnd2lzZURpc3AoZGdlLCBkZXNpZ24pCmZpdCA8LSBnbG1GaXQoZGdlLCBkZXNpZ24pCmxydCA8LSBnbG1MUlQoZml0LCBjb2VmPTIpCnJlcyA8LSByZXNpZHVhbHMoZml0LCB0eXBlPSJkZXZpYW5jZSIpCnNldCA8LSBuZXdTZXFFeHByZXNzaW9uU2V0KGRnZSRjb3VudHMsIHBoZW5vRGF0YT1kZ2Ukc2FtcGxlcykKZm9yKGsgaW4gMToxMCkgewogIHJ1dnJfc2V0IDwtIFJVVnIoc2V0LCByb3cubmFtZXMoZGdlKSwgaz1rLCByZXMpCiAgYXNzYXkocmVhZHNfc2NlLCBwYXN0ZSgiUlVWciBrPSIsIHRvU3RyaW5nKGspKSkgPC0gbG9nMih0KHQoYXNzYXlEYXRhKHJ1dnJfc2V0KSRub3JtYWxpemVkQ291bnRzKSAvIGNvbFN1bXMoYXNzYXlEYXRhKHJ1dnJfc2V0KSRub3JtYWxpemVkQ291bnRzKSAqIDFlNikgKyAxKQp9CmZvcihuIGluIGFzc2F5TmFtZXMocmVhZHNfc2NlKSkgewogIHByaW50KAogICAgICAgIHBsb3RQQ0EoCiAgICAgICAgICAgIHJlYWRzX3NjZSwKICAgICAgICAgICAgY29sb3VyX2J5ID0gIkxpYnJhcnkuR2VuZXJhdGlvbi5TZXQiLAogICAgICAgICAgICBzaXplX2J5ID0gInRvdGFsX2ZlYXR1cmVzIiwKICAgICAgICAgICAgc2hhcGVfYnkgPSAiU291cmNlIiwKICAgICAgICAgICAgZXhwcnNfdmFsdWVzID0gbgogICAgICAgICkgKyBnZ3RpdGxlKG4pCiAgKQp9CmBgYAojIyBEZXRlY3Qgb3V0bGllcnMKYGBge3J9CnJlYWRzX3NjZSA8LSBydW5QQ0EocmVhZHNfc2NlLCB1c2VfY29sZGF0YSA9IFRSVUUsIGRldGVjdF9vdXRsaWVycyA9IFRSVUUpCm91dGxpZXJzIDwtIGNvbG5hbWVzKHJlYWRzX3NjZSlbcmVhZHNfc2NlJG91dGxpZXJdCmhlYWQob3V0bGllcnMpCmBgYAojIyBFeGFtaW5lIHNvdXJjZXMgb2YgdmFyaWFuY2UgYWZ0ZXIgcmVtb3ZpbmcgdW53YW50ZWQgdmFyaWF0aW9uCmBgYHtyfQpmb3IgKHZhciBpbiBjKCJ0b3RhbF9mZWF0dXJlcyIsICJBZ2UiLCAiTGlicmFyeS5HZW5lcmF0aW9uLlNldCIsICJJbmRleCIsICJQYXJ0aWNpcGFudC5JRCIsICJTb3VyY2UiLCAiUmFjZSIsICJHZW5kZXIiLCAiQ29sbGVjdGlvbi5EYXRlIikpIHsKICBwcmludCgKICAgIHBsb3RRQyhyZWFkc19zY2UsIHR5cGUgPSAiZmluZC1wY3MiLCBleHByc192YWx1ZXMgPSAiUlVWciBrPSAyIiwgdmFyaWFibGUgPSB2YXIpCiAgICApICArIGdndGl0bGUodmFyKSAKfQpgYGAKIyMgVmlzdWFsaXplIHRvcCBwbGFzbWEtdnMtc2VydW0gbWlSTkFzCmBgYHtyfQpsaWJyYXJ5KFJDb2xvckJyZXdlcikKbGlicmFyeShyZXNoYXBlMikKIyBSeWFuJ3MgY29kZSwgbW9kaWZpZWQKbm9ybS5leHByLm1hdHIgPC0gZXhwcnMocmVhZHNfc2NlKQojIFJhbmsgdGhlIG1lYW4gZXhwcmVzc2lvbiB2YWx1ZXMgZm9yIHBsYXNtYS9zZXJ1bSBtaVJzLiBIaWdoZXN0IGV4cHJlc3Npb24gPSAxCm1lYW4uZXhwci5wbGFzbWEucmFuayA8LSByYW5rKC0xKnJvd01lYW5zKG5vcm0uZXhwci5tYXRyWywgY29sRGF0YShyZWFkc19zY2UpJFNvdXJjZT09IlBsYXNtYSJdKSkKbWVhbi5leHByLnNlcnVtLnJhbmsgPC0gcmFuaygtMSpyb3dNZWFucyhub3JtLmV4cHIubWF0clssIGNvbERhdGEocmVhZHNfc2NlKSRTb3VyY2U9PSJTZXJ1bSJdKSkKdG9wX04gPC0gMTggICMgdG9wX049MTggcmVzdWxzIGluIDIwIG1pUk5BcyBpbiB0aGUgcGxvdAp0b3AubWlScyA8LSByb3cubmFtZXMobm9ybS5leHByLm1hdHIpW21lYW4uZXhwci5wbGFzbWEucmFuayA8PSB0b3BfTiB8IG1lYW4uZXhwci5zZXJ1bS5yYW5rIDw9IHRvcF9OXSAjIGdldCB0aGUgbmFtZXMgb2YgdGhlIHRvcCBtaVJzIGluIHBsYXNtYSBvciBzZXJ1bQpub3JtLmV4cHIudG9wIDwtIG5vcm0uZXhwci5tYXRyW3RvcC5taVJzLCBdICMgR2V0IHRoZSBleHByZXNzaW9uIG1hdHJpeCBmb3IgdGhlIHRvcCBtSVJzCm5vcm0uZXhwci5tZWx0IDwtIHJlc2hhcGUyOjptZWx0KG5vcm0uZXhwci50b3ApICMgQ29udmVydCB5b3VyIG5vcm1hbGl6ZWQgZXhwcmVzc2lvbiBtYXRyaXggdG8gYSAzIGNvbHVtbiBkYXRhLmZyYW1lIChyb3cubmFtZSwgY29sLm5hbWUsIGV4cHJlc3Npb24gdmFsdWUpLiBNZWx0IGlzIGluIHRoZSBkcHlsciBwYWNrYWdlLCBJIGJlbGlldmUuCmNvbG5hbWVzKG5vcm0uZXhwci5tZWx0KSA8LSBjKCJtaVIuSUQiLCAiTVQuVW5pcXVlLklEIiwgIm5vcm0uZXhwciIpICMganVzdCBzbyBpdCdzIGVhc2llciBmb3IgbWUgdG8gdGVsbCB5b3Ugd2hpY2ggY29sdW1ucyBJJ20gdXNpbmcuCm5vcm0uZXhwci5tZWx0JFNvdXJjZSA8LSAiIgpmb3IgKHJvd19udW0gaW4gMTpucm93KG5vcm0uZXhwci5tZWx0KSl7ICAjIFB1bGwgdGhlIHBsYXNtYS9zZXJ1bSBzb3VyY2UgdmFsdWUgZnJvbSB0aGUgY29sdW1uIG1ldGFkYXRhLgogIG10X3VuaXF1ZV9pZCA8LSBub3JtLmV4cHIubWVsdFtyb3dfbnVtLCBdJE1ULlVuaXF1ZS5JRAogIG5vcm0uZXhwci5tZWx0W3Jvd19udW0sICJTb3VyY2UiXSA8LSBhcy5jaGFyYWN0ZXIoc2FtcGxlc1ttdF91bmlxdWVfaWQsIlNvdXJjZSJdKQp9CiMgVGhpcyB3b3VsZCBiZSBhIHNpbXBsZSBib3hwbG90IHdpdGggcGxhc21hL3NlcnVtIHNpZGUtYnktc2lkZS4gT3ZlcmxheWluZyB0aGUgYm94ZXMgb3ZlciBwb2ludHMgdGFrZXMgYSBsaXR0bGUgbW9yZSB0d2Vha2luZyB0byBnZXQgdGhlIGRvZGdlL3dpZHRoIHJpZ2h0LCBidXQgaXQncyBkb2FibGUuCmdncGxvdChub3JtLmV4cHIubWVsdCwgYWVzKHg9cmVvcmRlcihtaVIuSUQsIG5vcm0uZXhwciwgRlVOPW1lZGlhbiksIHk9bm9ybS5leHByLCBmaWxsPVNvdXJjZSkpICsgZ2VvbV9ib3hwbG90KHBvcz0iZG9kZ2UiLCBvdXRsaWVyLnNpemU9MC41KSArIAogIGdndGl0bGUoIlRvcCAyMCBFeHByZXNzZWQgbWlSTkFzIikgKyB5bGFiKCJOb3JtYWxpemVkIEV4cHJlc3Npb24iKSArIHhsYWIoIm1pUk5BIElEIikgKyAKICB0aGVtZShwYW5lbC5ncmlkLm1ham9yLnggPSBlbGVtZW50X2xpbmUoc2l6ZSA9IDAuNSwgbGluZXR5cGUgPSAnc29saWQnLCBjb2xvdXIgPSAiZ3JleSIpLCAKICAgICAgICBwYW5lbC5ncmlkLm1ham9yLnkgPSBlbGVtZW50X2JsYW5rKCksCiAgICAgICAgYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA5MCwgaGp1c3QgPSAxKSkKZ2dwbG90KG5vcm0uZXhwci5tZWx0LCBhZXMoeD1yZW9yZGVyKG1pUi5JRCwgbm9ybS5leHByLCBGVU49bWVkaWFuKSwgeT1ub3JtLmV4cHIsIGZpbGw9U291cmNlKSkgKyBnZW9tX2JveHBsb3QocG9zPSJkb2RnZSIsIG91dGxpZXIuc2l6ZT0wLjUpICsgCiAgZ2d0aXRsZSgiVG9wIDIwIEV4cHJlc3NlZCBtaVJOQXMiKSArIHlsYWIoIk5vcm1hbGl6ZWQgRXhwcmVzc2lvbiIpICsgeGxhYigibWlSTkEgSUQiKSArIAogIHRoZW1lKHBhbmVsLmdyaWQubWFqb3IueSA9IGVsZW1lbnRfbGluZShzaXplID0gMC41LCBsaW5ldHlwZSA9ICdzb2xpZCcsIGNvbG91ciA9ICJncmV5IiksIAogICAgICAgIHBhbmVsLmdyaWQubWFqb3IueCA9IGVsZW1lbnRfYmxhbmsoKSkgKyBjb29yZF9mbGlwKCkKYGBgCgojIFZpc3VhbGl6ZSBFZGdlUiByZXN1bHRzCmBgYHtyfQp0b3BUYWdzKGxydCkKIyBUT0RPOiBmaWx0ZXIgYnkgcC12YWx1ZSBhbmQvb3IgRkRSCiMgVE9ETzogaGVhdG1hcApgYGAKCiMgUnVuIHRoZSBERVNlcTIgcGlwZWxpbmUgJiBWaXN1YWxpemUgcmVzdWx0cwpgYGB7cn0KbGlicmFyeShERVNlcTIpICAjIFRPRE86IHVzZSBmaWx0ZXJlZCAmIFJVVlNlcS1jb3JyZWN0ZWQgZGF0YQpkZXNlcTJEYXRhIDwtIERFU2VxRGF0YVNldEZyb21NYXRyaXgoY291bnREYXRhID0gY291bnRzLCBkZXNpZ24gPSB+IFBhcnRpY2lwYW50LklEICsgU291cmNlLCBjb2xEYXRhID0gc2FtcGxlcykKZGVzZXEyRGF0YSA8LSBERVNlcShkZXNlcTJEYXRhKQpyZXN1bHRzIDwtIHJlc3VsdHMoZGVzZXEyRGF0YSwgY29va3NDdXRvZmY9RkFMU0UsIGluZGVwZW5kZW50RmlsdGVyaW5nPUZBTFNFKQpoZWFkKGFzLmRhdGEuZnJhbWUocmVzdWx0cykpCmBgYAoKIyMgQ3JlYXRlIGEgaGVhdG1hcCBvZiB0aGUgdmFyaWFuY2Utc3RhYmlsaXppbmctdHJhbnNmb3JtZWQgZXhwcmVzc2lvbiBkYXRhCmBgYHtyfQpsaWJyYXJ5KHBoZWF0bWFwKSAgICMgVE9ETzogb25seSB1c2UgdG9wIERFIG1pUk5Bcwp2c3REYXRhIDwtIHZhcmlhbmNlU3RhYmlsaXppbmdUcmFuc2Zvcm1hdGlvbihkZXNlcTJEYXRhLCBibGluZD1GQUxTRSkgICMgdHJhbnNmb3JtIHRoZSBkYXRhCnNlbGVjdCA8LSBvcmRlcihyb3dNZWFucyhjb3VudHMoZGVzZXEyRGF0YSxub3JtYWxpemVkPVRSVUUpKSwgZGVjcmVhc2luZz1UUlVFKQpTb3VyY2UgPC0gY29sRGF0YShkZXNlcTJEYXRhKVssYygiU291cmNlIildCmFubm90YXRpb25zIDwtIGFzLmRhdGEuZnJhbWUoU291cmNlKQpyb3duYW1lcyhhbm5vdGF0aW9ucykgPC0gY29sbmFtZXMoZGVzZXEyRGF0YSkKcGhlYXRtYXAoYXNzYXkodnN0RGF0YSlbc2VsZWN0LF0sIGNsdXN0ZXJfcm93cz1UUlVFLCBzaG93X3Jvd25hbWVzPUZBTFNFLCBjbHVzdGVyX2NvbHM9VFJVRSwgYW5ub3RhdGlvbl9jb2w9YW5ub3RhdGlvbnMpCmBgYAoKIyMgQ3JlYXRlIGEgaGVhdG1hcCBvZiBzYW1wbGUtdG8tc2FtcGxlIGRpc3RhbmNlcwpgYGB7cn0Kc2FtcGxlRGlzdHMgPC0gZGlzdCh0KGFzc2F5KHZzdERhdGEpKSkgICMgY29tcHV0ZSBzYW1wbGUtdG8tc2FtcGxlIGRpc3RhbmNlcwpzYW1wbGVEaXN0TWF0cml4IDwtIGFzLm1hdHJpeChzYW1wbGVEaXN0cykKcm93bmFtZXMoc2FtcGxlRGlzdE1hdHJpeCkgPC0gcGFzdGUodnN0RGF0YSRQYXJ0aWNpcGFudC5JRCwgdnN0RGF0YSRTb3VyY2UsIHNlcD0iIC0gIikKY29sbmFtZXMoc2FtcGxlRGlzdE1hdHJpeCkgPC0gTlVMTApjb2xvcnMgPC0gY29sb3JSYW1wUGFsZXR0ZSggcmV2KGJyZXdlci5wYWwoOSwgIkJsdWVzIikpICkoMjU1KQpwaGVhdG1hcChzYW1wbGVEaXN0TWF0cml4LAogICAgICAgICBjbHVzdGVyaW5nX2Rpc3RhbmNlX3Jvd3M9c2FtcGxlRGlzdHMsCiAgICAgICAgIGNsdXN0ZXJpbmdfZGlzdGFuY2VfY29scz1zYW1wbGVEaXN0cywKICAgICAgICAgY29sPWNvbG9ycykKYGBgCgojIyBDcmVhdGUgYSBQQ0EgcGxvdApgYGB7cn0KcGNhRGF0YSA8LSBwbG90UENBKHZzdERhdGEsIGludGdyb3VwPWMoIlNvdXJjZSIpLCByZXR1cm5EYXRhPVRSVUUpCnBlcmNlbnRWYXIgPC0gcm91bmQoMTAwICogYXR0cihwY2FEYXRhLCAicGVyY2VudFZhciIpKQpnZ3Bsb3QocGNhRGF0YSwgYWVzKFBDMSwgUEMyLCBjb2xvcj1Tb3VyY2UpKSArCiAgZ2VvbV9wb2ludChzaXplPTMpICsKICB4bGFiKHBhc3RlMCgiUEMxOiAiLHBlcmNlbnRWYXJbMV0sIiUgdmFyaWFuY2UiKSkgKwogIHlsYWIocGFzdGUwKCJQQzI6ICIscGVyY2VudFZhclsyXSwiJSB2YXJpYW5jZSIpKSArIAogIGNvb3JkX2ZpeGVkKCkKYGBgCmBgYHtyfQpzYXZlLmltYWdlKCJkaWZmX2V4cHJfcGxhc21hX3ZzX3NlcnVtLlJEYXRhIikKYGBgCgo=